{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyalink.alink import *\n",
    "useLocalEnv(1)\n",
    "\n",
    "from utils import *\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "DATA_DIR = ROOT_DIR + \"wine\" + os.sep\n",
    "\n",
    "ORIGIN_FILE = \"winequality-white.csv\";\n",
    "\n",
    "TRAIN_FILE = \"train.ak\";\n",
    "TEST_FILE = \"test.ak\";\n",
    "\n",
    "COL_NAMES = [\n",
    "    \"fixedAcidity\", \"volatileAcidity\", \"citricAcid\", \"residualSugar\", \"chlorides\",\n",
    "    \"freeSulfurDioxide\", \"totalSulfurDioxide\", \"density\", \"pH\", \"sulphates\",\n",
    "    \"alcohol\", \"quality\"\n",
    "]\n",
    "\n",
    "COL_TYPES = [\n",
    "    \"double\", \"double\", \"double\", \"double\", \"double\",\n",
    "    \"double\", \"double\", \"double\", \"double\", \"double\",\n",
    "    \"double\", \"double\"\n",
    "]\n",
    "\n",
    "FEATURE_COL_NAMES = COL_NAMES.copy()\n",
    "FEATURE_COL_NAMES.remove(\"quality\")\n",
    "\n",
    "LABEL_COL_NAME = \"quality\";\n",
    "PREDICTION_COL_NAME = \"pred\";\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_1\n",
    "source = CsvSourceBatchOp()\\\n",
    "    .setFilePath(DATA_DIR + ORIGIN_FILE)\\\n",
    "    .setSchemaStr(generateSchemaString(COL_NAMES, COL_TYPES))\\\n",
    "    .setFieldDelimiter(\";\")\\\n",
    "    .setIgnoreFirstLine(True);\n",
    "\n",
    "source.lazyPrint(5);\n",
    "\n",
    "source.link(CorrelationBatchOp().lazyPrintCorrelation());\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "corr = source.collectToDataframe().corr()  \n",
    "plt.figure(figsize=(15, 5))\n",
    "sns.heatmap(corr, annot = True, cmap=\"Greys\") ;\n",
    "\n",
    "source\\\n",
    "    .groupBy(LABEL_COL_NAME, LABEL_COL_NAME + \", COUNT(*) AS cnt\")\\\n",
    "    .orderBy(LABEL_COL_NAME, 100)\\\n",
    "    .lazyPrint(-1);\n",
    "\n",
    "BatchOperator.execute();\n",
    "\n",
    "splitTrainTestIfNotExist(source, DATA_DIR + TRAIN_FILE, DATA_DIR + TEST_FILE, 0.8);\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_2\n",
    "train_data = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_FILE);\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE);\n",
    "\n",
    "LinearRegression()\\\n",
    "    .setFeatureCols(FEATURE_COL_NAMES)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .enableLazyPrintTrainInfo()\\\n",
    "    .enableLazyPrintModelInfo()\\\n",
    "    .fit(train_data)\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalRegressionBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"LinearRegression\")\n",
    "    );\n",
    "\n",
    "LassoRegression()\\\n",
    "    .setLambda(0.05)\\\n",
    "    .setFeatureCols(FEATURE_COL_NAMES)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .enableLazyPrintTrainInfo()\\\n",
    "    .enableLazyPrintModelInfo(\"< LASSO model >\")\\\n",
    "    .fit(train_data)\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalRegressionBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"LassoRegression\")\n",
    "    );\n",
    "\n",
    "BatchOperator.execute();\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_3\n",
    "train_data = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_FILE);\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE);\n",
    "\n",
    "DecisionTreeRegressor()\\\n",
    "    .setFeatureCols(FEATURE_COL_NAMES)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .fit(train_data)\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalRegressionBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"DecisionTreeRegressor\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "\n",
    "for numTrees in [2, 4, 8, 16, 32, 64, 128] :\n",
    "    RandomForestRegressor()\\\n",
    "        .setNumTrees(numTrees)\\\n",
    "        .setFeatureCols(FEATURE_COL_NAMES)\\\n",
    "        .setLabelCol(LABEL_COL_NAME)\\\n",
    "        .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "        .fit(train_data)\\\n",
    "        .transform(test_data)\\\n",
    "        .link(\n",
    "            EvalRegressionBatchOp()\\\n",
    "                .setLabelCol(LABEL_COL_NAME)\\\n",
    "                .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "                .lazyPrintMetrics(\"RandomForestRegressor - \" + str(numTrees))\n",
    "        )\n",
    "    BatchOperator.execute();\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_4\n",
    "train_data = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_FILE);\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE);\n",
    "\n",
    "for numTrees in [16, 32, 64, 128, 256, 512] :\n",
    "    GbdtRegressor()\\\n",
    "        .setLearningRate(0.05)\\\n",
    "        .setMaxLeaves(256)\\\n",
    "        .setFeatureSubsamplingRatio(0.3)\\\n",
    "        .setMinSamplesPerLeaf(2)\\\n",
    "        .setMaxDepth(100)\\\n",
    "        .setNumTrees(numTrees)\\\n",
    "        .setFeatureCols(FEATURE_COL_NAMES)\\\n",
    "        .setLabelCol(LABEL_COL_NAME)\\\n",
    "        .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "        .fit(train_data)\\\n",
    "        .transform(test_data)\\\n",
    "        .link(\n",
    "            EvalRegressionBatchOp()\\\n",
    "                .setLabelCol(LABEL_COL_NAME)\\\n",
    "                .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "                .lazyPrintMetrics(\"GbdtRegressor - \" + str(numTrees))\n",
    "        );\n",
    "    BatchOperator.execute()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
